import asyncio

from com.arcfox.base.base_spider import BaseSpider, ExecuteType
from com.arcfox.middleware.async_redis_middleware import open_redis
from com.arcfox.school.processor.rank.xiaoyou_school_processor import XiaoYouSchoolProcessor
from com.arcfox.util import async_request as requests


class XiaoYouSchoolSpider(BaseSpider):
    '''
    校友会学校排名数据抓取
    '''
    def __init__(self):
        self.url = "http://www.chinaxy.com/2022index/news/news.jsp"
        self.processor = XiaoYouSchoolProcessor()

    @open_redis
    async def init_data_version(self, client):
        return await super().init_data_version(client)

    async def _pull_task(self):
        # 2022 - 2019
        return [
                   {"year": "2022", "id": 1929},
                   {"year": "2021", "id": 52},
                   {"year": "2020", "id": 709},
                   {"year": "2019", "id": 707}
               ], ExecuteType.ONCE

    async def _crawl_by_task(self, tasks):
        for task in tasks:
            url = f"{self.url}?information_id={task['id']}"
            resp = await requests.get_with_session(url, headers=self.get_default_header())
            if resp.code == 200:
                await self.processor.parse_and_save_data(task, resp.response)


if __name__ == "__main__":
    asyncio.run(XiaoYouSchoolSpider().start_crawl())
